package WebSearch;
import org.jsoup.Jsoup;
import org.jsoup.helper.Validate;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.jsoup.HttpStatusException;

import dbAdapter.DatabaseAdapter;
import CommonClass.IMG;
import CommonClass.URLSearch;
import CommonClass.URLlink;
import CommonClass.keyword;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
import java.net.URLConnection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Vector;
import java.util.regex.*;
import java.util.zip.GZIPInputStream;
public class WebSearchV3 {
	String uRL;
	Document doc ;
	Elements links;
	Elements media;
	
	public String formatDomain(String domain)
	{
		Pattern httpPattern = Pattern.compile("^https?\\://");
        Pattern partialPattern = Pattern.compile("www\\.[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*)?");
        Pattern lpartialPattern = Pattern.compile("[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*)?");
        Pattern ipPattern = Pattern.compile("([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}");
        Pattern fullipPattern = Pattern.compile("^https?\\://([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}");
        Matcher ipMatcher = fullipPattern.matcher(domain);
        Pattern fullURLpattern = Pattern.compile("^https?\\://(www\\.)?[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*)?");
        Matcher matcher = fullURLpattern.matcher(domain);
        
        if(httpPattern.matcher(domain).find())
        {
        	return domain;
        }
        if(matcher.matches())
        {
        	//URL == http://www.google.com
        	return domain;
        }
        else if(partialPattern.matcher(domain).find())
        {
        	// URL == www.google.com
        	String uRLStr = "http://" + domain;
        	// Check with full URL again
        	if(matcher.reset(uRLStr).matches())
        		return uRLStr;
        	else
        		return null;
        }

        
        
        if(ipMatcher.matches())
        {
        	return domain;
        }
        else if(ipPattern.matcher(domain).find())
        {
        	String uRLStr = "http://" + domain;
        	if(ipMatcher.reset(uRLStr).find())
        		return uRLStr;
        	else
        		return null;
        }
        else if(lpartialPattern.matcher(domain).find())
        {
        	// URL == "google.com"
        	String uRLStr = "http://" + domain;
        	// Check with full URL again
        	if(matcher.reset(uRLStr).matches())
        		return uRLStr;
        	else
        		return null;
        }
		return domain;

	}
	
	/*
	 *  Add all the IMAGE and Keywords to Database
	 *  Return the IMAGE that has the Keywords (DONE)
	 */
	public ArrayList<IMG> getIMGs(String[] keyword, int size, DatabaseAdapter db) throws Exception
	{
		ArrayList<IMG> listOfIMG = new ArrayList<IMG>();
		Vector<String> UrlVector = new Vector<String>();
		Vector<IMG>	imageVector = new Vector<IMG>();
		Vector<keyword> keywordVector = new Vector<keyword>();
		Pattern imgPattern = Pattern.compile("\\.(jp(s|g|eg)|png|bmp|gif|raw)");
        media = doc.body().select("[src]");
		for (Element src : media) 
		{
			
			String altStr = src.attr("alt");
			String titleStr = src.attr("title");
			String keywords = new String();
			String tags = new String();
			boolean isKeywordMatch = false;
			boolean singleRun1 = false;
			boolean singleRun2 = false;
			//addResult(String URL, IMG image, keyword _keyword){
			
			 for(int i = 0; i < size; i++)
			 {
				 if(!imgPattern.matcher(src.attr("abs:src").toLowerCase().toString()).find())
				 {
					 break;
				 }
				 if(src.attr("abs:src").toLowerCase().contains(keyword[i].toLowerCase()))
				 {
					 isKeywordMatch = true;
				 }
				 if(altStr.isEmpty() && titleStr.isEmpty())
				 {
					 break;
				 }
				 if(!altStr.isEmpty())
				 {
					 if(altStr.toLowerCase().contains(keyword[i].toLowerCase()))
					 {
						 isKeywordMatch = true;
					 }
					 if(!singleRun1)
					 {
						 singleRun1 = true;
						 tags = "ALT ";
						 keywords = "< " + altStr.toLowerCase() + ">, ";
					 }
				 }
				 if(!titleStr.isEmpty())
				 {
					 if(titleStr.toLowerCase().contains(keyword[i].toLowerCase()))
					 {
						 isKeywordMatch = true;

					 }
					 if(!singleRun2)
					 {
						 singleRun2 = true;
						 tags += "TITLE";
						 keywords += "< " + titleStr.toLowerCase() + ">";
					 }
				 }

			 }
			 
			 String qq = src.attr("abs:src");
			 if(isKeywordMatch)
			 {
				 //IMG(String uRL, String IMG, int height, int width)
				 listOfIMG.add(new IMG( src.attr("abs:src"), 0, 0));
			 }
			 //addResult(String URL, IMG image, keyword _keyword){
			 //keyword(String uRL, String tag, String keyword)
			 UrlVector.add(uRL);
			 imageVector.add(new IMG( src.attr("abs:src"), 0, 0)); 
			 keywordVector.add(new keyword( tags, keywords));
			 
		} 
		 db.addResultList(UrlVector, imageVector, keywordVector);
		 return listOfIMG;
	} 
	
	public void getHREFs(String[] keyword, int size, DatabaseAdapter db) throws Exception
	{
		Pattern redirectPattern = Pattern.compile(uRL+"#");
		 //ArrayList<URLlink> listOfURL = new ArrayList<URLlink>();
		 links = doc.body().select("a[href]");
		 //int ss = links.size();
		 HashMap<URLlink, Integer> mpp = new HashMap<URLlink, Integer>();
		for (Element link : links) 
		{
			String altStr = link.attr("alt");
			String textStr = link.text();
			String titleStr = link.attr("title");
			int c = 0;
			if(!redirectPattern.matcher(link.attr("abs:href")).find()){
			for(int i = 0; i < size; i++)
			{
				if(link.attr("abs:href").toLowerCase().contains(keyword[i].toLowerCase()))
				{
					c++;
				}
				if(altStr.isEmpty() && titleStr.isEmpty() && textStr.isEmpty())
				{
					break;
				}
				if(!altStr.isEmpty())
				{
					 if(altStr.toLowerCase().contains(keyword[i].toLowerCase()))
					 {
						 c++;
					 }
				}
				if(!titleStr.isEmpty())
				{
					 if(titleStr.toLowerCase().contains(keyword[i].toLowerCase()))
					 {
						 c++;
					 }
				}
				if(!textStr.isEmpty())
				{
					 if(textStr.toLowerCase().contains(keyword[i].toLowerCase()))
					 {
						 c++;
					 }
				}				
			}
			URLlink tmp1 = new URLlink(link.attr("abs:href"));
			Integer i2 = new Integer(c);
			mpp.put(tmp1, i2);
			
			}
			
		}
		db.addNewUnprocessedUrlList(mpp);
		//return null;
	} 
	private String getURL_Page(String URLStr) throws Exception
	{		
        URL anyURL = new URL(URLStr);
        URLConnection yc = anyURL.openConnection();
//        BufferedReader in = new BufferedReader(
//                                new InputStreamReader(
//                                yc.getInputStream()));

      GZIPInputStream gis = new GZIPInputStream(yc.getInputStream());
      BufferedReader bf = new BufferedReader(new InputStreamReader(gis, "UTF-8"));
      String outStr = "";
      String line;
      while ((line=bf.readLine())!=null) {
        outStr += line;
      }
     // System.out.println("Output String lenght : " + outStr.length());       
		return outStr;	

	}
	private String getWeb(String uRL, int port) throws Exception
	{
		
		Pattern tagPattern = Pattern.compile("^\\s*$");
		Pattern EncodeFile = Pattern.compile(".*Content-Encoding.*gzip.*");
		Pattern Error300 = Pattern.compile(" 30\\d ");
		Pattern Error500 = Pattern.compile(" 50\\d ");
		String URLStr = uRL;
		
		while(true)
		{
			int getContentLength;
			URL newURL = new URL(URLStr);
			InetAddress address = InetAddress.getByName(newURL.getHost());
			String ip = address.getHostAddress();
			
			Socket s = new Socket();
			s.connect(new InetSocketAddress(ip, 80), 2000 + port);

			PrintWriter pw = new PrintWriter(s.getOutputStream());
			pw.println("GET "+ newURL.getPath() +" HTTP/1.0");
			pw.println("Accept: text/plain");
			//pw.println("Accept-Encoding: gzip;q=0, compress;q=0");
			pw.println("Accept-Language: en");
			pw.println("Host: "+newURL.getHost()+"\r\n");
//			System.out.println("GET "+ newURL.getPath() +" HTTP/1.0\r\n");
//			System.out.println("Host: "+ newURL.getHost() +"\r\n\r\n");
			pw.println();
			pw.flush();
			InputStreamReader isr = new InputStreamReader(s.getInputStream(), "UTF-8");
			BufferedReader br = new BufferedReader(isr);
			StringBuilder builder = new StringBuilder();
			String HTTPheader = new String();
			String ContentLength = new String();
			String str;
			while((str = br.readLine()) != null)
			{
				if(tagPattern.matcher(str).matches())
				{
					break;
				}
				//System.out.println(str);
				//HTTPheader += str + "<>";
				builder.append(str+"<>");
			}
			HTTPheader = builder.toString();
			if(Error300.matcher(HTTPheader).find())
			{
				int i = HTTPheader.indexOf("Location: ");
				if(i > -1){
					int j = HTTPheader.indexOf("<>", i+10);
					URLStr = HTTPheader.substring(i+10, j);
				}
				else
				{
					s.close();
					throw new IOException("300 Exception");
				}
			}
			if(HTTPheader.indexOf("200 O") > -1)
			{
				
				getContentLength = HTTPheader.indexOf("Content-Length: ");
				if(getContentLength > -1)
				{
					int j = HTTPheader.indexOf("<>", getContentLength+16);
					ContentLength = HTTPheader.substring(getContentLength+16, j);
				}
				if(EncodeFile.matcher(HTTPheader).find())
				{
					s.close();
					return getURL_Page(URLStr);
				}
				//int value;
//				String strstr = new String();
//				while((str = br.readLine()) != null)
//				{
//					//System.out.println(str);
//					strstr += str;
//				}
				//builder.delete(0, builder.length());
				StringBuilder builder1 = new StringBuilder();
				char[] chars = new char[20*1024];
				int len;
				while((len = br.read(chars)) >=0)
				{
					builder1.append(chars, 0, len);
				}
		         s.close();
		         return builder1.toString();

			}
			if(HTTPheader.indexOf("404 Not Found") > -1)
			{
				s.close();
				throw new IOException("Exception 404");
				
			}
			if(Error500.matcher(HTTPheader).find())
			{
				s.close();
				throw new IOException("500 Exception");
			}
			s.close();
			//break;
		}
		//return uRL;
		
	}	
	public URLSearch fetchWebPageWithKeyword(String uRL, String keyword, int port)
	{
		 //Validate.isTrue(uRL == null || uRL.length() == 0, "usage: supply url to fetch");
		// Validate.isFalse(checkURL(uRL), "usage: supply a URL with in the format http://www.example.com");
		 System.out.println("Fetching " + uRL + " ...");
		 this.uRL = uRL;
		 try {
				//doc= Jsoup.connect(uRL).get();
			 String webpage = getWeb(uRL,port);
			 doc = Jsoup.parse(webpage);
			DatabaseAdapter db = DatabaseAdapter.getInstance();
			
			//Getting Mutex
			
				String[] keyword_array = keyword.split(" ");
				ArrayList<IMG> listOfImg = getIMGs(keyword_array, keyword_array.length, db);
				getHREFs(keyword_array, keyword_array.length, db);
				
				return  new URLSearch(uRL, null, listOfImg);
		

				

		} catch (IOException e) {
			// TODO Auto-generated catch block
			
			e.printStackTrace();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
	}
	
	public DatabaseAdapter fetchWebPageV1(String uRL, int port) throws Exception
	{
		 //Validate.isTrue(uRL == null || uRL.length() == 0, "usage: supply url to fetch");
		// Validate.isFalse(checkURL(uRL), "usage: supply a URL with in the format http://www.example.com");
		 System.out.println("Fetching " + uRL + " ...");
		 this.uRL = uRL;
		
			//doc= Jsoup.connect(uRL).get();
			 String webpage = getWeb(uRL, port);
			 doc = Jsoup.parse(webpage, uRL);
			DatabaseAdapter db = DatabaseAdapter.getInstance();
			
			//Getting Mutex
			
			
			return db;
//				String[] keyword_array = keyword.split(" ");
//				ArrayList<IMG> listOfImg = getIMGs(keyword_array, keyword_array.length, db);
//				getHREFs(keyword_array, keyword_array.length, db);
//				db.returnMutex();
//				return  new URLSearch(uRL, null, listOfImg);
		

				

		
		//return null;
		
	}
	public void fetchWebPage(String uRL) throws SearchException
	{
		 //Validate.isTrue(uRL == null || uRL.length() == 0, "usage: supply url to fetch");
		// Validate.isFalse(checkURL(uRL), "usage: supply a URL with in the format http://www.example.com");
		 System.out.println("Fetching " + uRL + " ...");
		 this.uRL = uRL;
		 try {
			doc= Jsoup.connect(uRL).get();
	        links = doc.body().select("a[href]");
	        media = doc.body().select("img[src]");
	        //ArrayList<String> t = new ArrayList<String>();
	        System.out.printf("\nMedia: (%d)", media.size());
//	        for (Element src : media) {
//	            if (src.tagName().equals("img"))
//	            {
//	            	System.out.printf(" * %s: <%s> %sx%s (%s)\n",
//	                        src.tagName(), src.attr("abs:src"), src.attr("width"), src.attr("height"),
//	                        src.attr("alt"));
//	            	
//	            }
//	            else
//	            {
//	            	System.out.printf(" * %s: <%s>\n", src.tagName(), src.attr("abs:src"));
//	            }
//	            
//	        }
	        int z = 0;
	        Pattern redirectPattern = Pattern.compile(uRL+"#");
	        System.out.printf("\nLinks with #: (%d)", links.size());
	        for (Element link : links) {
	        	if(!redirectPattern.matcher(link.attr("abs:href")).find())
	        	{
	        		z++;
	        	//System.out.printf(" * a: <%s>  (%s) (%s) (%s) \n", link.attr("abs:href"), link.text(), link.attr("alt"), link.attr("title"));
	        	}//System.out.printf(" * a: <%s>  (%s) \n", link.attr("abs:href"), link.text());
	        }
	        System.out.printf("\nNo of link without #: (%d)", z);
			if (links != null)
			{
				links.clear();
				media.clear();
			}

				//return new URLSearch(uRL, getHREFs(), getIMGs());
		
		} catch (HttpStatusException e) {
			System.out.println((e.getUrl().toString()));
			e.printStackTrace();
			
			throw new SearchException(e.getStatusCode());
		}
		 catch (IOException e) {
			// TODO Auto-generated catch block
			System.out.println((e.getStackTrace())[0].toString());
			e.printStackTrace();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			System.out.println(e.getMessage());
			e.printStackTrace();
		}
		//return null;
	}

}
